{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost Parameter Tuning for Otto Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第二步：调整树的参数：max_depth & min_child_weight\n",
    "(粗调，参数的步长为2；下一步是在粗调最佳参数周围，将步长降为1，进行精细调整)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先 import 必要的模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# path to where the data lies\n",
    "dpath = './'\n",
    "train = pd.read_csv(dpath +\"FE_train.csv\")\n",
    "#train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variable Identification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Target 分布，看看各类样本分布是否均衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEKCAYAAADaa8itAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAFZVJREFUeJzt3X+wJWV95/H3h5+igoAOFJmBzCCzKpHS4AQhyfoLRTAb+VGSQKmwht3JuqjERFc0rrOKRNhERGo1LrUQQCNI8NdIkFlEMRtLgeGXCEgxCwoTiKAgIi7owHf/OM+FMzN37u2502cux/t+VZ263U8/3ed7rJFPdT/dT6eqkCSpD1vMdgGSpF8fhookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN1vNdgGb23Oe85xauHDhbJchSWPjmmuu+XFVzevSd86FysKFC1m5cuVslyFJYyPJD7v29fKXJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN3PuifpN9ZJ3nzfbJegp6Jq/Pma2S5CeEjxTkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9WakoZLknUluSvK9JOcneVqSRUmuTHJbks8l2ab13batr2rbFw4d572t/dYkrx1qP7i1rUpy4ih/iyRpeiMLlSTzgXcAS6rqhcCWwFHAqcDHqmox8ABwXNvlOOCBqtoL+FjrR5K9236/BRwMfDLJlkm2BD4BHALsDRzd+kqSZsmoL39tBWyXZCvg6cA9wKuAi9r2c4HD2vKhbZ22/cAkae0XVNWjVXUHsArYr31WVdXtVfVL4ILWV5I0S0YWKlX1L8DfAHcyCJMHgWuAn1bVmtZtNTC/Lc8H7mr7rmn9nz3cvs4+G2qXJM2SUV7+2onBmcMi4DeAZzC4VLWumthlA9s2tn2yWpYmWZlk5X333Tdd6ZKkGRrl5a9XA3dU1X1V9SvgC8DvAju2y2EAC4C72/JqYHeAtv1ZwP3D7evss6H29VTVmVW1pKqWzJs3r4/fJkmaxChD5U5g/yRPb2MjBwI3A98A3tD6HAt8uS0vb+u07V+vqmrtR7W7wxYBi4GrgKuBxe1usm0YDOYvH+HvkSRNY2Qv6aqqK5NcBFwLrAGuA84E/hG4IMmHW9tZbZezgE8nWcXgDOWodpybklzIIJDWAMdX1WMASd4GrGBwZ9nZVXXTqH6PJGl6I33zY1UtA5at03w7gzu31u37CHDkBo5zMnDyJO2XAJdseqWSpD74RL0kqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN9OGSpLnJtm2Lb8iyTuS7Dj60iRJ46bLmcrngceS7AWcBSwCPjvSqiRJY6lLqDxeVWuAw4HTq+qdwG6jLUuSNI66hMqvkhwNHAtc3Nq2Hl1JkqRx1SVU3gIcAJxcVXckWQR8ZrRlSZLG0VZTbUyyJfC+qnrTRFtV3QGcMurCJEnjZ8ozlap6DJiXZJvNVI8kaYxNeabS/AD4VpLlwMMTjVV12qiKkiSNpy6hcnf7bAFsP9pyJEnjbNpQqaoPAiR5RlU9PF1/SdLc1eWJ+gOS3Azc0tZflOSTI69MkjR2utxSfDrwWuAnAFV1A/CyURYlSRpPnSaUrKq71ml6bAS1SJLGXJdQuSvJ7wKVZJsk76JdCptOkh2TXJTk+0luaZfSdk5yWZLb2t+dWt8kOSPJqiTfTbLv0HGObf1vS3LsUPtLktzY9jkjSTby90uSetQlVP4TcDwwH1gNvLitd/Fx4NKqej7wIgZhdCJweVUtBi5v6wCHAIvbZynwtwBJdgaWAS8F9gOWTQRR67N0aL+DO9YlSRqBLnd//Rh448YeOMkODMZe/n07zi+BXyY5FHhF63YucAXwHuBQ4LyqKuA77Sxnt9b3sqq6vx33MuDgJFcAO1TVt1v7ecBhwFc3tlZJUj+mDZU219fbgYXD/avq9dPsuidwH/B3SV4EXAOcAOxaVfe0Y9yTZJfWfz4wPHazurVN1b56knZJ0izp8vDjlxi8R+UrwOMbeex9gbdX1ZVJPs6Tl7omM9l4SM2gff0DJ0sZXCZjjz32mKpmSdIm6BIqj1TVGTM49mpgdVVd2dYvYhAqP0qyWztL2Q24d6j/7kP7L2DwJP9qnrxcNtF+RWtfMEn/9VTVmcCZAEuWLJk0eCRJm67LQP3Hkyxrd27tO/GZbqeq+lcGd449rzUdCNwMLGfwbhba3y+35eXAMe0usP2BB9tlshXAQUl2agP0BwEr2raHkuzf7vo6ZuhYkqRZ0OVMZR/gzcCrePLyV7X16bwd+Ps2y/HtDN7NsgVwYZLjgDuBI1vfS4DXAauAX7S+VNX9SU4Crm79PjQxaA+8FTgH2I7BAL2D9JI0i7qEyuHAnu3urY1SVdcDSybZdOAkfYsN3KpcVWcDZ0/SvhJ44cbWJUkajS6Xv24Adhx1IZKk8dflTGVX4PtJrgYenWjscEuxJGmO6RIqy0ZehSTp10KXJ+q/mWRX4Hda01VVde9U+0iS5qYu71P5I+AqBndp/RFwZZI3jLowSdL46XL56y+B35k4O0kyD/gag4cZJUl6Qpe7v7ZY53LXTzruJ0maY7qcqVyaZAVwflv/Y3zIUJI0iS4D9e9OcgTw+wwmcTyzqr448sokSWOny9T3p1bVe4AvTNImSdITuoyNvGaStkP6LkSSNP42eKaS5K3Afwb2TPLdoU3bA98adWGSpPEz1eWvzzIYkP8Ia79c66GhWYIlSXrCBi9/VdWDVfUD4P3Av1bVD4FFwJuSOMGkJGk9XcZUPg88lmQvBq8VXsTgLEaSpLV0CZXHq2oNcARwelW9E9httGVJksZRl1D5VZKjGbyu9+LWtvXoSpIkjasuofIW4ADg5Kq6I8ki4DOjLUuSNI66PFF/M/COofU7gFNGWZQkaTx1eaL+DqDWba+qPUdSkSRpbHWZUHLJ0PLTGLxXZefRlCNJGmfTjqlU1U+GPv9SVacDr9oMtUmSxkyXy1/7Dq1uweDMZfuRVSRJGltdLn99dGh5DXAHg9cKS5K0li53f71ycxQiSRp/046pJPmr4bm+kuyU5MOjLUuSNI66PPx4SFX9dGKlqh4AXje6kiRJ46pLqGyZZNuJlSTbAdtO0V+SNEd1Gaj/DHB5kr9j8BDknwDnjrQqSdJY6jJQ/9/bmx9fDQQ4qapWjLwySdLY6XKmQlVdClw64lokSWOuy5iKJEmdGCqSpN5sMFSSXN7+nrr5ypEkjbOpxlR2S/Jy4PVJLmAwSP+Eqrp2pJVJksbOVKHyAeBEYAFw2jrbCmcqliStY4OhUlUXARcl+a9VddJmrEmSNKa6PKdyUpLXAy9rTVdU1cWjLUuSNI66TCj5EeAE4Ob2OaG1SZK0li4PP/4B8OKqehwgybnAdcB7R1mYJGn8dH1OZceh5WdtzBck2TLJdUkubuuLklyZ5LYkn0uyTWvftq2vatsXDh3jva391iSvHWo/uLWtSnLixtQlSepfl1D5CHBdknPaWco1wF9txHecANwytH4q8LGqWgw8ABzX2o8DHqiqvYCPtX4k2Rs4Cvgt4GDgky2otgQ+ARwC7A0c3fpKkmbJtKFSVecD+wNfaJ8DquqCLgdPsoDB5bP/1dbD4Fbki1qXc4HD2vKhPDn78UXAga3/ocAFVfVoVd0BrAL2a59VVXV7Vf0SuKD1lSTNkq4TSt4DLJ/B8U8H/guwfVt/NvDTqlrT1lcD89vyfOCu9n1rkjzY+s8HvjN0zOF97lqn/aWTFZFkKbAUYI899pjBz5AkdTGyub+S/Dvg3qq6Zrh5kq41zbaNbV+/serMqlpSVUvmzZs3RdWSpE3R6Uxlhn6PwRQvrwOeBuzA4MxlxyRbtbOVBcDdrf9qYHdgdZKtGNwQcP9Q+4ThfTbULkmaBVOeqSTZIsn3ZnLgqnpvVS2oqoUMBtq/XlVvBL4BvKF1Oxb4clte3tZp279eVdXaj2p3hy0CFgNXAVcDi9vdZNu075jJJTpJUk+mPFOpqseT3JBkj6q6s6fvfA9wQZIPM3je5azWfhbw6SSrGJyhHNVquCnJhQwevFwDHF9VjwEkeRuwAtgSOLuqbuqpRknSDHS5/LUbcFOSq4CHJxqr6vVdv6SqrgCuaMu3M7hza90+jwBHbmD/k4GTJ2m/BLikax2SpNHqEiofHHkVkqRfC10mlPxmkt8EFlfV15I8ncHlJkmS1tJlQsn/yOBhxP/ZmuYDXxplUZKk8dTlOZXjGdwe/DOAqroN2GWURUmSxlOXUHm0TYMCQHuGZNKHDCVJc1uXUPlmkvcB2yV5DfAPwFdGW5YkaRx1CZUTgfuAG4E/ZXAL7/tHWZQkaTx1ufvr8Tbl/ZUMLnvd2p50lyRpLdOGSpI/AD4F/F8GkzguSvKnVfXVURcnSRovXR5+/CjwyqpaBZDkucA/AoaKJGktXcZU7p0IlOZ24N4R1SNJGmMbPFNJckRbvCnJJcCFDMZUjmQwQ7AkSWuZ6vLXHw4t/wh4eVu+D9hpZBVJksbWBkOlqt6yOQuRJI2/Lnd/LQLeDiwc7r8xU99LkuaGLnd/fYnBC7S+Ajw+2nIkSeOsS6g8UlVnjLwSSdLY6xIqH0+yDPjfwKMTjVV17ciqkiSNpS6hsg/wZuBVPHn5q9q6JElP6BIqhwN7Dk9/L0nSZLo8UX8DsOOoC5Ekjb8uZyq7At9PcjVrj6l4S7EkaS1dQmXZyKuQJP1a6PI+lW9ujkIkSeOvyxP1D/HkO+m3AbYGHq6qHUZZmCRp/HQ5U9l+eD3JYcB+I6tIkjS2utz9tZaq+hI+oyJJmkSXy19HDK1uASzhycthkiQ9ocvdX8PvVVkD/AA4dCTVSJLGWpcxFd+rIknqZKrXCX9giv2qqk4aQT2SpDE21ZnKw5O0PQM4Dng2YKhIktYy1euEPzqxnGR74ATgLcAFwEc3tJ8kae6ackwlyc7AnwNvBM4F9q2qBzZHYZKk8TPVmMpfA0cAZwL7VNXPN1tVkqSxNNXDj38B/AbwfuDuJD9rn4eS/GzzlCdJGidTjals9NP2kqS5zeCQJPXGUJEk9WZkoZJk9yTfSHJLkpuSnNDad05yWZLb2t+dWnuSnJFkVZLvJtl36FjHtv63JTl2qP0lSW5s+5yRJKP6PZKk6Y3yTGUN8BdV9QJgf+D4JHsDJwKXV9Vi4PK2DnAIsLh9lgJ/C0/c1rwMeCmDKfeXTQRR67N0aL+DR/h7JEnTGFmoVNU9VXVtW34IuAWYz2AyynNbt3OBw9ryocB5NfAdYMckuwGvBS6rqvvbMzKXAQe3bTtU1berqoDzho4lSZoFm2VMJclC4LeBK4Fdq+oeGAQPsEvrNh+4a2i31a1tqvbVk7RP9v1Lk6xMsvK+++7b1J8jSdqAkYdKkmcCnwf+rKqmer5lsvGQmkH7+o1VZ1bVkqpaMm/evOlKliTN0EhDJcnWDALl76vqC635R+3SFe3vva19NbD70O4LgLunaV8wSbskaZaM8u6vAGcBt1TVaUOblgMTd3AdC3x5qP2YdhfY/sCD7fLYCuCgJDu1AfqDgBVt20NJ9m/fdczQsSRJs6DLmx9n6veANwM3Jrm+tb0POAW4MMlxwJ3AkW3bJcDrgFXALxjMiExV3Z/kJODq1u9DVXV/W34rcA6wHfDV9pEkzZKRhUpV/TOTj3sAHDhJ/wKO38CxzgbOnqR9JfDCTShTktQjn6iXJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1ZuxDJcnBSW5NsirJibNdjyTNZVvNdgGbIsmWwCeA1wCrgauTLK+qm2e3Mml23PmhfWa7BD0F7fGBGzfbd437mcp+wKqqur2qfglcABw6yzVJ0pw17qEyH7hraH11a5MkzYKxvvwFZJK2Wq9TshRY2lZ/nuTWkVY1dzwH+PFsF/FUkL85drZL0Pr89zlh2WT/qdwov9m147iHympg96H1BcDd63aqqjOBMzdXUXNFkpVVtWS265Am47/P2THul7+uBhYnWZRkG+AoYPks1yRJc9ZYn6lU1ZokbwNWAFsCZ1fVTbNcliTNWWMdKgBVdQlwyWzXMUd5SVFPZf77nAWpWm9cW5KkGRn3MRVJ0lOIoaIZcXocPVUlOTvJvUm+N9u1zEWGijba0PQ4hwB7A0cn2Xt2q5KecA5w8GwXMVcZKpoJp8fRU1ZV/RNw/2zXMVcZKpoJp8eRNClDRTPRaXocSXOPoaKZ6DQ9jqS5x1DRTDg9jqRJGSraaFW1BpiYHucW4EKnx9FTRZLzgW8Dz0uyOslxs13TXOIT9ZKk3nimIknqjaEiSeqNoSJJ6o2hIknqjaEiSeqNoSJ1lOSxJNcnuSnJDUn+PMkWbduSJGdMse8rkly8+apd7/v/W5J3zdb3a+4Y+zc/SpvR/6uqFwMk2QX4LPAsYFlVrQRWjuqLk2zVng+SntI8U5FmoKruBZYCb8vAE2ciSV7ezmiuT3Jdku3bbjsk+WKSm5N8augs5+cTx03yhiTntOVzkpyW5BvAqRs6bpJ3J7k6yXeTfHDoWH/Z3nnzNeB5m+N/F8kzFWmGqur2Fgy7rLPpXcDxVfWtJM8EHmnt+zF4/8wPgUuBI4CLpvmafwO8uqoeS/KVdY+b5CBgcTt2gOVJXgY8zGD6nN9m8P/za4FrNu0XS9PzTEXaNJPN2Pwt4LQk7wB2HLpsdVV7B81jwPnA73c4/j+0/hs67kHtcx2D4Hg+g5D5t8AXq+oXVfUznJtNm4mhIs1Qkj2Bx4B7h9ur6hTgPwDbAd9J8vyJTescoiZpf9o6fR6e5rgBPlJVL26fvarqrA18nzRyhoo0A0nmAZ8C/ketM4FekudW1Y1VdSqDwfuJUNmvzey8BfDHwD+39h8leUFrP3yK75zsuCuAP2mXw0gyv91E8E/A4Um2a2Mvf9jXb5em4piK1N12Sa4HtgbWAJ8GTpuk358leSWDs5ibga8CBzCYOfcUYB8G/9H/Yut/InAxg7dpfg945ga+f73jVtWjSV4AfDsJwM+BN1XVtUk+B1zPYAzn/2zKD5e6cpZiSVJvvPwlSeqNoSJJ6o2hIknqjaEiSeqNoSJJ6o2hIknqjaEiSeqNoSJJ6s3/B01v63PB5joIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(train.Disbursed);\n",
    "pyplot.xlabel('Disbursed');\n",
    "pyplot.ylabel('Number of customers');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "每类样本分布不是很均匀，所以交叉验证时也考虑各类样本按比例抽取"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "由于电脑老旧，故取1%的数据用于完成训练测试任务，集中精力放在参数调优的学习上。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "y = train['Disbursed']\n",
    "X = train.drop(['Disbursed'], axis=1)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.99,random_state=4)\n",
    "\n",
    "# drop ids and get labels\n",
    "y_train = y_train\n",
    "X_train = X_train.drop(['ID','DOB','LoggedIn','Unnamed: 26'], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "各类样本不均衡，交叉验证是采用StratifiedKFold，在每折采样时各类样本按比例采样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第一轮参数调整得到的n_estimators最优值（58），其余参数继续默认值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "用交叉验证评价模型性能时，用scoring参数定义评价指标。评价指标是越高越好，因此用一些损失函数当评价指标时，需要再加负号，如neg_log_loss，neg_mean_squared_error 详见sklearn文档：http://scikit-learn.org/stable/modules/model_evaluation.html#log-loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(3, 10, 2), 'min_child_weight': range(1, 6, 2)}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:762: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.07802, std: 0.01075, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.07813, std: 0.00822, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.08140, std: 0.00783, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.07743, std: 0.01028, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.07813, std: 0.00822, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.08140, std: 0.00783, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.07743, std: 0.01028, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.07813, std: 0.00822, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.08140, std: 0.00783, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.07743, std: 0.01028, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.07813, std: 0.00822, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.08140, std: 0.00783, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 5, 'min_child_weight': 1},\n",
       " -0.07743330438669514)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=58,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'binary:logistic',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.08367457, 0.06088014, 0.0546577 , 0.06990151, 0.05567179,\n",
       "        0.05296884, 0.07018061, 0.05569296, 0.05323071, 0.07635894,\n",
       "        0.05736837, 0.04517245]),\n",
       " 'std_fit_time': array([0.0060107 , 0.00463802, 0.00235535, 0.00119188, 0.00133662,\n",
       "        0.00068888, 0.00253002, 0.00091372, 0.00073348, 0.00687394,\n",
       "        0.0018723 , 0.00624441]),\n",
       " 'mean_score_time': array([0.00609102, 0.00431476, 0.00427322, 0.00445542, 0.00432963,\n",
       "        0.0041316 , 0.00443006, 0.00427837, 0.00421238, 0.00443177,\n",
       "        0.00422444, 0.00333357]),\n",
       " 'std_score_time': array([8.64092462e-04, 9.22906710e-05, 1.27467625e-04, 1.88721377e-04,\n",
       "        1.22209782e-04, 8.27515684e-05, 1.30883505e-04, 1.65162336e-04,\n",
       "        8.42298794e-05, 1.30156578e-04, 1.70657254e-04, 5.75763384e-04]),\n",
       " 'param_max_depth': masked_array(data=[3, 3, 3, 5, 5, 5, 7, 7, 7, 9, 9, 9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[1, 3, 5, 1, 3, 5, 1, 3, 5, 1, 3, 5],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 3, 'min_child_weight': 1},\n",
       "  {'max_depth': 3, 'min_child_weight': 3},\n",
       "  {'max_depth': 3, 'min_child_weight': 5},\n",
       "  {'max_depth': 5, 'min_child_weight': 1},\n",
       "  {'max_depth': 5, 'min_child_weight': 3},\n",
       "  {'max_depth': 5, 'min_child_weight': 5},\n",
       "  {'max_depth': 7, 'min_child_weight': 1},\n",
       "  {'max_depth': 7, 'min_child_weight': 3},\n",
       "  {'max_depth': 7, 'min_child_weight': 5},\n",
       "  {'max_depth': 9, 'min_child_weight': 1},\n",
       "  {'max_depth': 9, 'min_child_weight': 3},\n",
       "  {'max_depth': 9, 'min_child_weight': 5}],\n",
       " 'split0_test_score': array([-0.08134903, -0.07990878, -0.08576523, -0.08518067, -0.07990878,\n",
       "        -0.08576523, -0.08518067, -0.07990878, -0.08576523, -0.08518067,\n",
       "        -0.07990878, -0.08576523]),\n",
       " 'split1_test_score': array([-0.09588476, -0.08512757, -0.08730656, -0.09036689, -0.08512757,\n",
       "        -0.08730656, -0.09036689, -0.08512757, -0.08730656, -0.09036689,\n",
       "        -0.08512757, -0.08730656]),\n",
       " 'split2_test_score': array([-0.07391746, -0.08290783, -0.08396785, -0.07412736, -0.08290783,\n",
       "        -0.08396785, -0.07412736, -0.08290783, -0.08396785, -0.07412736,\n",
       "        -0.08290783, -0.08396785]),\n",
       " 'split3_test_score': array([-0.07587081, -0.0805199 , -0.08391115, -0.07695448, -0.0805199 ,\n",
       "        -0.08391115, -0.07695448, -0.0805199 , -0.08391115, -0.07695448,\n",
       "        -0.0805199 , -0.08391115]),\n",
       " 'split4_test_score': array([-0.06298407, -0.06209852, -0.06592263, -0.06039467, -0.06209852,\n",
       "        -0.06592263, -0.06039467, -0.06209852, -0.06592263, -0.06039467,\n",
       "        -0.06209852, -0.06592263]),\n",
       " 'mean_test_score': array([-0.07802233, -0.07813299, -0.08139749, -0.0774333 , -0.07813299,\n",
       "        -0.08139749, -0.0774333 , -0.07813299, -0.08139749, -0.0774333 ,\n",
       "        -0.07813299, -0.08139749]),\n",
       " 'std_test_score': array([0.01073802, 0.00820023, 0.00781184, 0.01027082, 0.00820023,\n",
       "        0.00781184, 0.01027082, 0.00820023, 0.00781184, 0.01027082,\n",
       "        0.00820023, 0.00781184]),\n",
       " 'rank_test_score': array([4, 5, 9, 1, 5, 9, 1, 5, 9, 1, 5, 9], dtype=int32),\n",
       " 'split0_train_score': array([-0.05315165, -0.07159667, -0.07875479, -0.05535395, -0.07159667,\n",
       "        -0.07875479, -0.05535395, -0.07159667, -0.07875479, -0.05535395,\n",
       "        -0.07159667, -0.07875479]),\n",
       " 'split1_train_score': array([-0.05959183, -0.0719274 , -0.07865269, -0.05985944, -0.0719274 ,\n",
       "        -0.07865269, -0.05985944, -0.0719274 , -0.07865269, -0.05985944,\n",
       "        -0.0719274 , -0.07865269]),\n",
       " 'split2_train_score': array([-0.05590089, -0.07510369, -0.07748212, -0.05709953, -0.07510369,\n",
       "        -0.07748212, -0.05709953, -0.07510369, -0.07748212, -0.05709953,\n",
       "        -0.07510369, -0.07748212]),\n",
       " 'split3_train_score': array([-0.05691435, -0.07237225, -0.07508728, -0.05588616, -0.07237225,\n",
       "        -0.07508728, -0.05588616, -0.07237225, -0.07508728, -0.05588616,\n",
       "        -0.07237225, -0.07508728]),\n",
       " 'split4_train_score': array([-0.05785779, -0.0757041 , -0.08098254, -0.05746041, -0.0757041 ,\n",
       "        -0.08098254, -0.05746041, -0.0757041 , -0.08098254, -0.05746041,\n",
       "        -0.0757041 , -0.08098254]),\n",
       " 'mean_train_score': array([-0.0566833 , -0.07334082, -0.07819189, -0.0571319 , -0.07334082,\n",
       "        -0.07819189, -0.0571319 , -0.07334082, -0.07819189, -0.0571319 ,\n",
       "        -0.07334082, -0.07819189]),\n",
       " 'std_train_score': array([0.00214369, 0.00171294, 0.00192142, 0.00156598, 0.00171294,\n",
       "        0.00192142, 0.00156598, 0.00171294, 0.00192142, 0.00156598,\n",
       "        0.00171294, 0.00192142])}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.077433 using {'max_depth': 5, 'min_child_weight': 1}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xd4FWX2wPHvSQihhBqKVOkdEiACiiBVwAIiLbgiIoqNdXddXSy7tnVV1FUsrD9RlKJSFTcq0kRFsdA29BaQElronZByfn/cSbyENEImk3I+z3Mf5868951zJ3LPnXfmnldUFWOMMSa3BXgdgDHGmMLJEowxxhhXWIIxxhjjCkswxhhjXGEJxhhjjCsswRhjjHGFJRhjjDGusARjjDHGFZZgjDHGuKKY1wF4qVKlSlqnTh2vwzDGmAJl5cqVh1S1clbtXE0wItIbeAMIBN5X1ZfSbA8GpgBtgcPAEFXdISJBwPtAGyfGKar6ovOaD4CbgDhVbeHX1zPAPcBBZ9UTqjo3s/jq1KnDihUrLvt9GmNMUSIiO7PTzrUhMhEJBMYDfYBmwFARaZam2UjgqKo2AF4HxjrrBwHBqtoSX/K5V0TqONsmAb0z2O3rqhruPDJNLsYYY9zl5jWYdkCMqm5X1fPAdKBfmjb9gMnO8mygu4gIoEBpESkGlATOAycAVHUJcMTFuI0xxuQCNxNMDWC33/NYZ126bVQ1ETgOhOJLNqeBfcAu4FVVzU5SGS0ia0TkAxGpcJnxG2OMuQxuXoORdNalnRsgozbtgCSgOlAB+EFEFqnq9kz29w7wT+f1/wT+Ddx1UVAio4BRALVr176ok4SEBGJjYzl37lwmuzKFRYkSJahZsyZBQUFeh2JMoeNmgokFavk9rwnszaBNrDMcVg7f8NdtwDxVTQDiRGQpEAFkmGBU9UDKsoi8B3yZQbsJwASAiIiIiybDiY2NpUyZMtSpUwffaJ0prFSVw4cPExsbS926db0Ox5hCx80hsuVAQxGpKyLFgUggKk2bKGC4szwQWKy+GdB2Ad3EpzTQAdiU2c5EpJrf0/7AupwEfe7cOUJDQy25FAEiQmhoqJ2tGuMS1xKMc01lNDAf2AjMVNX1IvKciPR1mk0EQkUkBngYeMxZPx4IwZcklgMfquoaABGZBvwMNBaRWBEZ6bzmZRFZKyJrgK7AX3IauyWXosP+1sa4x9XfwTi3Cs9Ns+4pv+Vz+G5JTvu6U+mtd7YNzWD9sMsK1hhjioCkxERmPdSLDn8cS52mEa7uy0rFGGNMETJrZCfCFu/ll/GPu74vSzD5zLFjx/jPf/6To9eOGzeOM2fO5HJEuaNLly45rprw+eefs2HDhkvq69y5c7Rr146wsDCaN2/O008/naN9G1OYTL+/G2G/HmNdq5IMGve16/uzBJPPFNYEcznSJpjsCA4OZvHixaxevZro6GjmzZvHL7/84lKExuR/0x+5mbBv97GpcRB9J/9IYDH3S1EW6WKXWXn2i/Vs2HsiV/tsVr0sT9/cPMPtjz32GNu2bSM8PJyePXtSpUoVZs6cSXx8PP379+fZZ5/l9OnTDB48mNjYWJKSkvjHP/7BgQMH2Lt3L127dqVSpUp8++236fYfEhLCgw8+yKJFi6hQoQIvvPACf/vb39i1axfjxo2jb9++7Nixg2HDhnH69GkA3n77ba655hrmzJnD+PHjWbhwIfv37+e6665jyZIlXHHFFRft5+zZs4wYMYINGzbQtGlTzp49m7ptwYIFPP3008THx1O/fn0+/PBDQkJCqFOnDkOGDEmN/ZNPPiEuLo6oqCi+//57nn/+eT799FMAZs2axQMPPMCxY8eYOHEinTp1umD/IkJISAjg+21TQkKCXdA3RdbsZ/5Ayy9jiKkbSM8p3xFcslSe7NfOYPKZl156ifr16xMdHU3Pnj3ZunUry5YtIzo6mpUrV7JkyRLmzZtH9erVWb16NevWraN379489NBDVK9enW+//TbD5AJw+vRpunTpwsqVKylTpgx///vfWbhwIXPmzOGpp3z3X1SpUoWFCxeyatUqZsyYwUMPPQRA//79ueKKKxg/fjz33HMPzz77bLrJBeCdd96hVKlSrFmzhieffJKVK1cCcOjQIZ5//nkWLVrEqlWriIiI4LXXXkt9XdmyZVm2bBmjR4/mz3/+M9dccw19+/bllVdeITo6mvr16wOQmJjIsmXLGDduHM8++ywAe/fu5YYbbkjtKykpifDwcKpUqULPnj1p3779ZfxljCmYPn/1ARrPXMWuGkLHSV8TUq5inu3bzmAykdmZRl5YsGABCxYsoHXr1gCcOnWKrVu30qlTJx555BHGjBnDTTfddNG398wUL16c3r19tUJbtmxJcHAwQUFBtGzZkh07dgC+b/yjR48mOjqawMBAtmzZkvr6t956ixYtWtChQweGDk33hj4AlixZkpqYWrVqRatWrQD45Zdf2LBhAx07dgTg/PnzXH311amvS+lz6NCh/OUvGd9pfuuttwLQtm3b1LirV6/O3Lm/37QYGBhIdHQ0x44do3///qxbt44WLVqk150xhdLXE56k7qRv2V8JWr0/m4pVa2X9olxkCSYfU1Uef/xx7r333ou2rVy5krlz5/L4449z/fXXp559ZCUoKCh1qCggIIDg4ODU5cTERABef/11qlatyurVq0lOTqZEiRKpr9+zZw8BAQEcOHCA5ORkAgIyPglOb0hKVenZsyfTpk3L8jWZDWmlxB0YGJgad0bKly9Ply5dmDdvniUYU2QsnvZvqrz9GUfLQr3xE6lRN20xe/fZEFk+U6ZMGU6ePAlAr169+OCDDzh16hTg+3CPi4tj7969lCpVittvv51HHnmEVatWXfTay3H8+HGqVatGQEAAU6dOJSkpCfANS40YMYJPPvmEpk2bXjC0lVbnzp35+OOPAVi3bh1r1qwBoEOHDixdupSYmBgAzpw5c8EZ0owZM1L/m3Jmk5P3dfDgQY4dOwb4rgctWrSIJk2aXFIfxhRUP331ISEvv8+5ElDx1Vep3/IaT+KwM5h8JjQ0lI4dO9KiRQv69OnDbbfdlvpBGxISwkcffURMTAyPPvooAQEBBAUF8c477wAwatQo+vTpQ7Vq1TK9DpOVBx54gAEDBjBr1iy6du1K6dKlAXjhhRfo1KkTnTp1Ijw8nKuuuoobb7yRpk2bXtTH/fffz4gRI2jVqhXh4eG0a9cOgMqVKzNp0iSGDh1KfHw8AM8//zyNGjUCID4+nvbt25OcnJx6lhMZGck999zDm2++yezZszOMe+/evdx9993MnTuXffv2MXz4cJKSkkhOTmbw4MHcdNNNOT4mxhQU0T/8F555GRUIev5JWl1zo2exiK/0V9EUERGhaX9PsXHjxnQ/MI37UmYYrVSpUp7u1/7mprDYHL2E/fffS8gZOPn3B+gy6I+u7EdEVqpqlmUAbIjMGGMKgd3b1hH7x3spdwoOPTTEteRyKWyIrJBq37596hBUiqlTp9KyZctc3c/8+fMZM2bMBevq1q3LnDlzLrmvlLvBjDGX5tC+31h/z2BqHIbf7ulJ35HPeB0SYAmm0Pr111/zZD+9evWiV69eebIvY8zFThyN45c7b6LuPmXT0KsY8Jc3vQ4plQ2RGWNMARV/9gyL7+hO/Z3JrO/bhAFPTfE6pAtYgjHGmAIoKTGRL4Z1pPHWRKJ71GDw2EsflnabJRhjjClgkhITmT38GpqvO8fqqysy9O1FXoeULlcTjIj0FpHNIhIjIo+lsz1YRGY4238VkTrO+iARmezMULlRRB73e80HIhInIuvS9FVRRBaKyFbnvxXcfG/GGOOVWfd3o9XKk6xtXZpB733vdTgZci3BiEggvqmP+wDNgKEikrZWwUjgqKo2AF4HxjrrBwHBqtoSaAvcm5J8gElA73R2+Rjwjao2BL7h9+mXC5TCWq4/r+eDAd/valq2bEl4eDgREe7O3GdMXpn+596E/XCQDU2L03/y0jwpu59Tbp7BtANiVHW7qp4HpgP90rTpB0x2lmcD3cVXgEqB0iJSDCgJnAdOAKjqEuBIOvvz72sycEsuvpc8U1gTzOXIyXwwKb799luio6NznNyMyU9mPjmYsHk72VI/kBs++oGg4sFeh5QpN1NfDWC33/NYIG299NQ2qpooIseBUHzJph+wDygF/EVV00sq/qqq6j6nr30iUuWy38HXj8H+tZfdzQWuaAl9Xspws80HkzvzwRhT2Hz24t00+2wt22sFcN2UhZQsXdbrkLLk5hlMeqVw09alyahNOyAJqA7UBf4qIvVyJSiRUSKyQkRWHDx4MDe6zFU2H0zuzQcjIlx//fW0bduWCRMmXMZfxRhvfTn+Uep/tJS9VYW2Ez+nfGg1r0PKFjfPYGIB/8kHagJ7M2gT6wyHlcM3/HUbME9VE4A4EVkKRADbM9nfARGp5py9VAPi0mukqhOACeCrRZbpO8jkTCMv2HwwlzcfzNKlS6levTpxcXH07NmTJk2a0Llz52wdJ2Pyi0VTXqD6u19ypDw0emcKV9Ru6HVI2eZmglkONBSRusAeIBJf4vAXBQwHfgYGAotVVUVkF9BNRD7CN0TWARiXxf5S+nrJ+e9/c+uNeMXmg7m8+WCqV68O+M7I+vfvz7JlyyzBmALlhznvUO61qZwuCVeMe4s6TQvWzSquDZGpaiIwGpgPbARmqup6EXlORPo6zSYCoSISAzzM73d+jQdCgHX4EtWHqroGQESm4UtIjUUkVkRGOq95CegpIluBns7zAsfmg8md+WBOnz6d+prTp0+zYMECm2zMFCgrv5lFseffJDEQSr3wHE2v6uF1SJfM1fvbVHUuMDfNuqf8ls/huyU57etOpbfe2ZbuuIyqHga6X068+YHNB5M788EcOHCA/v37A77EeNttt6UODRqT321cvogzTzxFiSQ49/RDXN093Y/DfM/mg7H5YPINmw/GGNixcQW/3T2McifhyCPD6HHHE16HdBGbD8YYYwqY/bu2suX+O6h4DPaOujFfJpdLkX9/Amoui80HY0zBcuzwPlaOvIUrDyhb7ujIraNf9Tqky2YJppCy+WCMKTjOnj7B93f0pNHuZNYOaMngx9/3OqRcYUNkxhjjoYTz8cy9vRONtiWxuveVDP7XTK9DyjWWYIwxxiNJiYnMGd6RZhvPs7pTZSLHzfM6pFxlCcYYYzwy657raPm/06xpW4ZB7yz2OpxcZwkmnyms1ZTzulz/5s2bCQ8PT32ULVuWceOyKgZhTN6ZPronYT8fYX3zYAZO/ilfl93PKUsw+UxhTTCXIyfl+hs3bkx0dHRqkdBSpUql/vDSGK/NHNOfsEWxbG5YjJs/KpzJBSzB5Dv+5fofffRRXnnlFa666ipatWrF008/DfhKn9x4442EhYXRokULZsyYwZtvvplarr9r164Z9h8SEsKYMWNo27YtPXr0YNmyZXTp0oV69eoRFRUF+G4X7tSpE23atKFNmzb89NNPAMyZM4cePXqgquzbt49GjRqxf//+dPdz9uxZIiMjadWqFUOGDLmoXP/VV19NmzZtGDRoUGopnDp16jBmzBjatWtHu3btiImJ4aeffiIqKopHH32U8PBwtm3bBvjK9bdr145GjRrxww8/ZHpMv/nmG+rXr8+VV16Zzb+CMe759Lk7aB61iW1XBtBtyjcElyzldUiuKZxpM5eMXTaWTUc25WqfTSo2YUy7MRluf+mll1i3bh3R0dEsWLCA2bNns2zZMlSVvn37smTJEg4ePEj16tX56quvAF/tsHLlyvHaa6/x7bffZvpL+JRy/WPHjqV///6p5fo3bNjA8OHD6du3b2q5/hIlSrB161aGDh3KihUr6N+/P59++injx49n3rx52S7Xv2bNGtq0aQNcWK6/dOnSjB07ltdeey21WGdKuf4pU6bw5z//mS+//JK+ffty0003MXDgwNT+U8r1z507l2effZZFixZdUCrG3/Tp0zOt/GxMXoka9ycaTV/O7mpCh0lfUrbC5U9blZ9ZgsnHrFz/5ZXrT+k/KiqKF198MctjY4yb5k98htoTFxAXCs3fm0mlanW9Dsl1lmAykdmZRl6wcv2XV64f4Ouvv6ZNmzZUrVo1wzbGuO27WW9R6c0ZHA+Bmm+9S636RaOyt12DyWesXH/ulOtPMW3aNBseM55atuAjSr74H+KLQ7mXX6JxeNGZk8jOYPIZK9efO+X6wZe8Fi5cyLvvvpvjY2HM5Vjz01ck/P1fBCnwzN8I79TP65DylJXrt3L9+YaV6zeFyba1P7H73pGUOQXHH7+bbkP/6nVIuSZflOsXkd4isllEYkTksXS2B4vIDGf7ryJSx1kfJCKTRWStiGwUkcez6lNEJonIbyIS7TzC3XxvxhiTkT2/bWD7gyOpcAIOPNC/UCWXS+HaEJmIBOKb+rgnEAssF5EoVfX/xdxI4KiqNhCRSGAsMATfbJbBqtpSREoBG5ypkndn0eejqprxGEoRYuX6jfHGkQO7WXP3QGoegm13deWW+17wOiTPuHkNph0Qo6rbAURkOtAP8E8w/YBnnOXZwNviu3VIgdIiUgwoCZwHTmSzT4OV6zfGC6eOH2HpnX2ot0fZGNmGgY/krCpHYeHmEFkNfGccKWKddem2UdVE4DgQii/ZnAb2AbuAV1X1SDb6/JeIrBGR10UkOBffizHGZCr+7BkW3tGFBr8lsfamBgx85mOvQ/KcmwkmvR8xpL2jIKM27YAkoDpQF/iriNTLos/HgSbAVUBFIN0fsYjIKBFZISIrDh48mOWbMMaYrCQlJhI1/FqabE5gdddqRL76hdch5QtuJphYoJbf85rA3ozaOMNh5YAjwG3APFVNUNU4YCkQkVmfqrpPfeKBD/ElqYuo6gRVjVDViMqVK1/mWzTGGJh917W0WHOW1e3LE1kIy+7nlJsJZjnQUETqikhxIBKIStMmChjuLA8EFqvvvuldQDfxKQ10ADZl1qeIVHP+K8AtwDoX35sxxgAw7b6utFp2nHVhJRk0MfPCq0WNawnGuaYyGpgPbARmqup6EXlORPo6zSYCoSISAzwMpNx2PB4IwZcklgMfquqajPp0XvOxiKwF1gKVgOfdem9uKqzl+vN6PhiAN954gxYtWtC8eXObC8a4YsZfbyL8u/1sbBxE30k/Ftqy+znl6u9gVHWuqjZS1fqq+i9n3VOqGuUsn1PVQaraQFXbpdwdpqqnnPXNVbWZqr6SWZ/O+m6q2lJVW6jq7ap6ys335pbCmmAuR07mg1m3bh3vvfcey5YtY/Xq1Xz55Zds3brVpQhNUTTrqaG0+GobW+sG0uujHwp12f2cslpk+YzNB5M788Fs3LiRDh06UKpUKYoVK8Z1112Xo9/mGJOez1+5jyazo9lVQ+g0ZT6ly5TzOqR8yc7nMrH/hReI35i788EEN23CFU88keF2mw8md+aDadGiBU8++SSHDx+mZMmSzJ07l4iILCtbGJOluf/3BHUnf8/+ykL4B59RoXLaX1+YFJZg8jGbDybn88E0bdqUMWPG0LNnT0JCQggLC6OYjY+by7T4k1e54j9zOFoW6r8ziWpXNvE6pHzN/sVlIrMzjbxg88Fc3nwwI0eOZOTIkQA88cQT1KxZM8P+jMnKT199SMgrEzlTAiq99hr1mqX7Swjjx67B5DM2H0zuzQcTFxcHwK5du/jss89sXhiTY//7fg48/TLJAsWff5IWHfp4HVKBYGcw+YzNB5N788EMGDCAw4cPExQUxPjx46lQoUKOj4kpujb/7ztOjnmCUglw5h8P0vn6270OqcCw+WBsPph8w+aDMfnNrq2r2XpXJKHH4MBfhtLrruwNRRd2+WI+GGOMKagO7fuNjaOGUvkI7L67lyWXHLAhskLK5oMxJudOHI3jl+E3UXe/sun29gz4k1WCyAlLMIWUzQdjTM6cPX2CxcO603hXMmtuacKQJyd5HVKBZUNk6SjK16WKGvtbG38J5+P56o7ONI5JZHXPWgx5yao/XA5LMGmUKFGCw4cP2wdPEaCqHD58+ILf+ZiiKykxkc9GXEvz9fGsvqYikW8t8DqkAs+GyNKoWbMmsbGx2GRkRUOJEiXsB5gGgFn3dSVs5SnWti7NoAnfex1OoWAJJo2goCDq1q3rdRjGmDw07U+9Cf/xEBuaBdN/8lIru59LbIjMGFOkzXxiIOHzd7KlfjFumLqEoOLBXodUaFiCMcYUWZ+9eDfN5qxne60ArpuygJKly3odUqHiaoIRkd4isllEYkTksXS2B4vIDGf7ryJSx1kfJCKTRWStiGwUkcez6tOZRvlXEdnq9FnczfdmjCnYvnzrrzT4aCl7qgptJ35O+dBqXodU6LiWYEQkEN/Ux32AZsBQEWmWptlI4KiqNgBeB8Y66wcBwaraEmgL3CsidbLocyzwuqo2BI46fRtjzEUWTHqeGhPmcqgCNH33I66o3dDrkAolN89g2gExqrpdVc8D04F+adr0AyY7y7OB7uKr0a5AaREpBpQEzgMnMurTeU03pw+cPm9x760ZYwqqH+a8Q8VxH3OqFFR/Yzy1G7fxOqRCy80EUwPY7fc81lmXbhtVTQSOA6H4EsVpYB+wC3hVVY9k0mcocMzpI6N9GWOKuBXfTKfY82+SUAxCXnqeJm27eR1SoebmvXjpzRaV9teLGbVpByQB1YEKwA8isiiT9tnZl2+HIqOAUQC1a9dON3BjTOGzcfkizj7xLCWSIP7Zv9C66wCvQyr03DyDiQVq+T2vCezNqI0zHFYOOALcBsxT1QRVjQOWAhGZ9HkIKO/0kdG+AFDVCaoaoaoRlStXvoy3Z4wpKHZsXMH+P/+R0mfh+F+Hc22/UV6HVCS4mWCWAw2du7uKA5FAVJo2UcBwZ3kgsFh9NVp2Ad3EpzTQAdiUUZ/Oa751+sDp878uvjdjTAGxf9dWttx3BxWPwd57b6LHsItuaDUucS3BONdDRgPzgY3ATFVdLyLPiUhfp9lEIFREYoCHgZS//HggBFiHL6l8qKprMurTec0Y4GGnr1Cnb2NMEXbs8D5WjbyF6nHK9mGduOnBV7wOqUixGS3TzGhpjCkcTp88zqIhHWm0PYm1A1oy+F8zvQ6p0LAZLY0xRVbC+Xjm39GZRtuTWN27jiUXj1iCMcYUKkmJicy54xqabjxPdOfKRI772uuQiixLMMaYQmXWPdfRMvoMayLKMPg/i70Op0izBGOMKTSmPdiDsJ+PsL5FCQZO+snK7nssywQjIvVFJNhZ7iIiD4lIefdDM8aY7Jvxt1sI/2YPmxoGcfNUm9MlP8jOGcynQJKINMB3629d4BNXozLGmEvw6XN30OKLzWy7MoCeH31HcMlSXodkyF6CSXZ+f9IfGKeqfwGsrrUxJl+Iev0hGk1fzu5qQodJXxJSrqLXIRlHdhJMgogMxffr+C+ddUHuhWSMMdkz//2nqT1xIQdCofl7M6lUzaY7z0+yk2BGAFcD/1LV30SkLvCRu2EZY0zmvpsxjkpvzeR4Gaj11rvUqt/C65BMGlleBVPVDcBDACJSASijqi+5HZgxxmTk13lTKDn2XeKLQ4VXX6ZxeGevQzLpyM5dZN+JSFkRqQisBj4UkdfcD80YYy625qevSHzqRd+TZ/5Gq443exuQyVB2hsjKqeoJ4FZ8RSfbAj3cDcsYYy62dfVSjjzyCCXOwZm/3cM1N47wOiSTiewkmGIiUg0YzO8X+Y0xJk/t+W0DO/94NxVOQNzoW+ka+bDXIZksZCfBPIevPP42VV0uIvWAre6GZYwxvztyYDdr7h5I1UOwY0Q3+oz6l9chmWzIzkX+WcAsv+fbAZtr1BiTJ04dP8LS4b2pt0fZGNmGgX8d73VIJpuyc5G/pojMEZE4ETkgIp+KSM28CM4YU7TFnz3Dwju60GBHMutuasjAZz72OiRzCbIzRPYhvqmNqwM1gC+cdVkSkd4isllEYkTkonlKRSRYRGY4238VkTrO+j+ISLTfI1lEwp1tQ0RkjYisF5GX/fq6U0QO+r3m7uzEaIzJn5ISE4m641qabE5gdbdqDHk17YzrJr/LToKprKofqmqi85gEVM7qRSISiG/q4z5AM2CoiDRL02wkcFRVGwCvA2MBVPVjVQ1X1XBgGLBDVaNFJBR4Beiuqs2BqiLS3a+/GSmvU9X3s/HejDH51Oy7OtJi7VlWty9PpJXdL5Cyk2AOicjtIhLoPG4HDmfjde2AGFXdrqrngelAvzRt+gGTneXZQHcRkTRthgLTnOV6wBZVPeg8X4RdDzKm0Jl2XxdaLTvBurCSDJr4g9fhmBzKToK5C98tyvuBfcBAfOVjslID2O33PNZZl24bp6DmcSA0TZsh/J5gYoAmIlJHRIoBtwC1/NoOcIbPZotILYwxBc70h28k/LsDbGxSnFum/mxl9wuwLBOMqu5S1b6qWllVq6jqLfh+dJmVtGciAHopbUSkPXBGVdc5sRwF7gdmAD8AO4BEp/kXQB1VbYXvzGYy6RCRUSKyQkRWHDx4ML0mxhiPzHpqKC3nbmdr3UB6TV1CUPFgr0MylyGnM1pm5xdOsVx4dlET2JtRG+eMpBxwxG97JL+fvQCgql+oantVvRrYjPObHFU9rKrxTrP3gLbpBaWqE1Q1QlUjKlfO8lKSMSaPfP7KfTSZHc3OmkKnKfMpXaac1yGZy5TTBJPemUday4GGIlJXRIrjSxZpbwOJwjcNAPiG3harqgKISAAwCN+1m993LFLF+W8F4AHgfee5/xw1fYGNl/KGjDHe+eqdx6k7+Xv2VxZaT/yMCpXTjqabgiing5tph7oubqCaKCKj8VUBCAQ+UNX1IvIcsEJVo/DNkDlVRGLwnblE+nXRGYh1ftjp7w0RCXOWn1PVLc7yQyLSF9+Q2RHgzhy+N2NMHvrm45ep9s7nHCkL9d+ZRLUrm3gdkskl4pwwXLxB5CTpJxIBSqpqgb/yFhERoStWrPA6DGOKrJ+/nEjgP17lfBCUf/M1WnTo43VIJhtEZKWqRmTVLsMkoaplcjckY4z53f++n4M+8yrJAsHPP2XJpRDK6TUYY4zJsc3/+46TY56geAIJcH8bAAAeSUlEQVScf+JBrrp+qNchGRdYgjHG5KldW1ez54/3U/Y0HP7TUDoPHO11SMYllmCMMXkmbs82No4aSqWjsPvuXvS66ymvQzIusgRjjMkTJ47GsWxEX2ruV7be1p6b/zTO65CMy7JTrv+kiJxI89jtlPCvlxdBGmMKtrOnT7B4WHfq70pmXb+mDHhyktchmTyQnVuNX8P3C/xP8N2iHAlcge9X9B8AXdwKzhhT8CWcj2fuHZ1pFpPI6p61iHzpM69DMnkkO0NkvVX1XVU9qaonVHUCcIOqzgAquByfMaYAS0pMZM6Ia2m2Pp7VHUOJfGuB1yGZPJSdBJMsIoNFJMB5DPbbluUv+o0xRdes+7rScuUp1rYuzaB3v/M6HJPHspNg/oBv0q845zEMuF1ESgJ2f6ExJl3TH+pF2I+H2NAsmP6Tl1rZ/SIoy7+4Uwvs5gw2/5i74RhjCoOZjw8gbMEuNjcoxg1Wdr/Iys5dZDWdO8biROSAiHwqIjXzIjhjTMHz2Qsjafb5BrbXDqDL5AWULF3W65CMR7IzRPYhvrL61fHNQPmFs84YYy7w5Vt/pcHHP7GnqnDVh1GUD62W9YtMoZWdBFNZVT9U1UTnMQmwmbqMMRdYMOmf1Jgwl0MVoOm7H1GlRn2vQzIey06COSQit4tIoPO4HTjsdmDGmILjh8/GU3HcJ5wsDdXfGE/txm28DsnkA9lJMHcBg4H9wD58M0+OcDMoY0zBseKb6QT9620SikGZF5+nSdtuXodk8oksE4yq7lLVvqpaWVWrqOotwK3Z6VxEeovIZhGJEZHH0tkeLCIznO2/ikgdZ/0fRCTa75EsIuHOtiEiskZE1ovIy1n1ZYxxz/pfF3Du8WcJSAZ96mFadx3gdUgmH8lpscuHs2ogIoHAeKAP0AwYKiLN0jQbCRxV1QbA68BYAFX9WFXDVTUc3+9udqhqtIiEAq8A3VW1OVBVRLpn1pcxxh07Nq4g7uE/UeocnHh4ONf0vcfrkEw+k9MEI9lo0w6IUdXtqnoemA70S9OmHzDZWZ4NdBeRtH0PBaY5y/WALap60Hm+CEj5ypSdvowxuWDfzk1sve8OKh6Dfff1pcewiwYojMlxgslOiZgawG6/57HOunTbqGoicBwITdNmCL8nmBigiYjUEZFiwC1ArUvoyxhzmY4e3MP/Rg6gWpyy/Y7O3PiADRaY9GX4S34ROUn6iUSAktnoO72zh7T9ZdpGRNoDZ1R1HYCqHhWR+4EZQDLwE76zmuzuDxEZBYwCqF27dhZvwRjj7/TJ4/wwvBf1Y5PZMLAVg8a863VIJh/L8AxGVcuoatl0HmVUNTtFhWL5/ewCoCa+sv/ptnHOSMoBR/y2R/L72UtKXF+oantVvRrflAFbs9lXyusnqGqEqkZUrmw/5zEmuxLOxzN/WGcabk9ibZ+6DHp+htchmXzOzRktlwMNRaSuiBTHlyyi0rSJAoY7ywOBxaqqACISAAzCd+0mlYhUcf5bAXgAeD+rvowxlycpMZHPh11D003nie5chcjX53odkikAXCtvqqqJIjIamA8EAh+o6noReQ5YoapRwERgqojE4DvbiPTrojMQ6xTb9PeGiIQ5y8+p6hZnObO+jDGXYdY9nQlbfYY1EWUZ/J9vvA7HFBBSlL/kR0RE6IoVK7wOw5h8bfoD3QlbvJf1LUrQf/pyK7tvEJGVqhqRVTs3h8iMMQXcjEf7EbZ4L5saBXHzVJvTxVwaSzDGmHTNfu4OWnyxhZg6AfSc+h3BJUt5HZIpYCzBGGMu8t9//5HG05ezu4bQcfI8QspV9DokUwBZgjHGXGDee//gyg8XcSAUmk+YScWqtbJ+kTHpsARjjEn13YxxVH57NsfLwJVvv0+t+i28DskUYJZgjDEA/DpvCiXHvsu54lDh1ZdpGNbR65BMAWcJxhjDmqVfkPiPFwEo9s/HadXxZo8jMoWBJRhjiritq5dy9JG/USIezvztHtr3vsPrkEwhYQnGmCJs97Z17Bx9N+VOwsE/DqRrZJZTPRmTbZZgjCmijhzYzfpRg6l6GHaO6EHve/7pdUimkLEEY0wRdOr4EZYO702tPcrmwW3p99e3vA7JFEKWYIwpYuLPnmHhsC402JHMupsbMfCZj7wOyRRSlmCMKUKSEhP5YlhHmmxJYHW36gx55b9eh2QKMUswxhQRSYmJzB7RkebrzrG6Q3kirey+cZklGGOKiFkP9qDV8hOsCyvFoPd/8DocUwRYgjGmCJj+8I2EfX+AjU2Kc8vUn6zsvskTriYYEektIptFJEZEHktne7CIzHC2/yoidZz1fxCRaL9HsoiEO9uGishaEVkjIvNEpJKz/hkR2eP3mhvcfG/GFBQz/xFJy7nb2VovkF5TlxBUPNjrkEwR4VqCEZFAYDzQB2gGDBWRZmmajQSOqmoD4HVgLICqfqyq4aoaDgwDdqhqtIgUA94AuqpqK2ANMNqvv9dTXqeqNmm4KfLmjL2XZrNXs7Om0GnyfEqXKed1SKYIcfMMph0Qo6rbVfU8MB3ol6ZNP2Cyszwb6C4ikqbNUGCasyzOo7TTriyw143gjSnovvrPGOpNWcK+KkLriZ9RoXINr0MyRYybCaYGsNvveayzLt02qpoIHAdC07QZgpNgVDUBuB9Yiy+xNAMm+rUd7QydfSAiFXLpfRhT4Hzz8ctU+78ojpSH+u9MotqVTbwOyRRBbiaYtGciAHopbUSkPXBGVdc5z4PwJZjWQHV8Q2SPO83fAeoD4cA+4N/pBiUySkRWiMiKgwcPZv/dGFNA/BT1HmVe/ZAzJaDKa29Qr1k7r0MyRZSbCSYW8J8KryYXD2eltnGur5QDjvhtj+T34THwJQ9UdZuqKjATuMZZd0BVk1Q1GXgP3xDdRVR1gqpGqGpE5cqVc/rejMmX/vftp8hzr5EcAMHPP0Xz9td7HZIpwtxMMMuBhiJSV0SK40sWUWnaRAHDneWBwGIncSAiAcAgfNduUuwBmolISmboCWx02lfza9cfWJeL78WYfG/TysWcfPzvBCVCwpOjuer6oV6HZIo4126GV9VEERkNzAcCgQ9Udb2IPAesUNUofNdPpopIDL4zl0i/LjoDsaq63a/PvSLyLLBERBKAncCdzuaXnVuZFdgB3OvWezMmv9m1eRV7//QgFU7Dwb/exvW3Puh1SMYgzglDkRQREaErVqzwOgxjLkvcnm387w83U+2gsvPePtz80Gteh2QKORFZqaoRWbWzX/IbU4CdOBrH8hF9qXlA2Xrb1ZZcTL5iCcaYAurs6RMsHtaderuSWX9LMwY8+YHXIRlzAUswxhRACefjmTusM41jElndsxaDX/zU65CMuYglGGMKmKTERObceS3NNsSzumMlIt9a4HVIxqTLEowxBcyse7vSctUp1rYJYdC733odjjEZsgRjTAEy/Y/XE7b0EBuaB9N/0o9Wdt/ka5ZgjCkgZjx2K2ELd7O5QTFumGJl903+ZwnGmALg03/dSYv/bmR77QC6Tf2GkqXLeh2SMVmyBGNMPvfFmw/T8JNfib1CuOrDKMpWqOJ1SMZkiyUYY/KxBZP+Sc33vuZQBWg6YRpVatT3OiRjss0SjDH51JLZb1Px9U84WRpqvPUOtRuGeR2SMZfEEowx+dDyBdMo/sJ4zgdBmbEv0Lh1F69DMuaS2T2OxuSyxKRkziYkcS4hmXMJSc4jmXOJvuWz55M4l+i/zdl+5iSlNs+hzPZfqL8iluIK+swjtL6uv9dvyZgcsQRjCr3kZCU+5QM90fmA9/vAP+eXDPwTQ3ya5ynL8SmJIr0kkpBEYnL2KpRXTdzDdWe+pdHRbdQ/eIoaB5TgBN+2Q+Xg5CMj6H7TSBePjDHusgRj8pyqkpCkv3/An/f/sE/59n/xh7//tvh0EsXZhGTiEy7+8I9PTM5xrCWDAikRFECJoEBKBgUSnPK8WCBVygSlLpcoHuj7r1/bEkEBTvtASgSAbl/EuVVfwZZtlN19miqHfGPUSQIHqgixbcpRtmUzGvS8jaZhPXLvgBvjEUswBvB9y/d9UGfvAz7lg9v3IZ9yZuD7sI9PN1H4fdNPTCYpm9/y0yoeGEBwUIDzAf77B3qJoEDKlQyiRJngCz7gS6S2S//D/4J+igVS0kkUwUEBBBcLQERyFOf5U8fYtOB99v38DUlb9lA2NoFyp33bzhSHuBqB7GxdlSrtrqZJr7tpUaVOjvZjTH7maoIRkd7AG/hmtHxfVV9Ksz0YmAK0BQ4DQ1R1h4j8AXjUr2kroI2qRovIUOAJfDNX7gVuV9VDIlIRmAHUwTej5WBVPerm+3OTqnI+KfmiIZgMP7Sdb/ApH/bxCc6Hf+KF2zL68D+flLNv+QGC34d24EUf/uVLBl3w7T69xHDRB3xQAMGpy4GUKPZ728CAnH3gu+3w9lVsnj+Jo/9bRdD2o1Tdl0zxJKgNHCoPh+qV4FSTutS67kbCOt1GseCSXodsjOtcm9FSRAKBLUBPIBZYDgxV1Q1+bR4AWqnqfSISCfRX1SFp+mkJ/FdV64lIMXxJpZmTVF4GzqjqM87yEVV9SUQeAyqo6pjMYszpjJan4xM5euZ8uuPvZ/2+pZ/z+3Z/wfBOFt/uUxJDTv80wakfyGmGdooFpH5DT+8DPsMP/5TXBfm91uknKFBy/C2/oEpOSmL70hns/O5zzm7YRpldZ6hyxLctMQD2VxXi65anfFhLGvYcRtVm13obsDG5LLszWrp5BtMOiFHV7U5A04F+wAa/Nv2AZ5zl2cDbIiJ6YdYbCkxzlsV5lBaRw0BZIMavry7O8mTgOyDTBJNTU37eydh5m7LdvliApPPB7ftALx1cjIqls/52n3Yo54KzgtTlQIKLBRCQT7/lF1Rnju5n8/z32f/rEnTrPirHJhJyDqoDp0rAwZrF2NG+Gle070zjXnfTssIVXodsTL7gZoKpAez2ex4LtM+ojaomishxIBQ45NdmCL7kgaomiMj9wFrgNLAVeNBpV1VV9znt9omIa/U0ujSuTGhI8Yy/3adJFEGB9nOjgiRu009smT+FY6vXEPzbMa44oJRI9o29xlWEA01KcaJZfa7scgttrh5IYFBxr0M2Jl9yM8Gk9zU67aBPpm1EpD2+IbB1zvMg4H6gNbAdeAt4HHg+20GJjAJGAdSuXTu7L7tA02plaVrNig0WBkkJ54n5fiq7v/+Kc5t2UG73WSod833LKRMIB6oFsOvailQIb03j3iNoWq+N1yEbU2C4mWBigVp+z2viu36SXptY5/pKOeCI3/ZIfh8eAwgHUNVtACIyE3jM2XZARKo5Zy/VgLj0glLVCcAE8F2DycH7MgXYqbgdbJr/PnHLfkZiDlBlbxKl4n2n0idKweFaQZzpVINqV3enSc8RhJUJ9TpkYwosNxPMcqChiNQF9uBLFrelaRMFDAd+BgYCi1Ouv4hIADAI6OzXfg/QTEQqq+pBfDcQbEzT10vOf//rxpsyBcve1YuIWTSNE2vWU3LHCarGKaUVrgTiKgn7WoRQunlj6nUfQOOIvgQEBnodsjGFhmsJxrmmMhqYj+825Q9Udb2IPAesUNUoYCIwVURi8J25RPp10RmITblJwOlzr4g8CywRkQRgJ3Cns/klYKaIjAR24UtOpghJOHuKrYsnE/vjfBI276L87ngqnoTKQNkg33DXzi6VCW0TQZM+d9O8ZlOvQzamUHPtNuWCIKe3KZv84fjezWz+eiKHVi4jYNshqu5NooRTauVoCBytHUxQ41rU6Hg9DbvdQfFS5bwN2JhCIj/cpmxMrklOSmL3yi/5bfFsTq3bROmdp6lyUCkDlBbYX0XY07oMZVo2p0GPSJq27uV1yMYUeZZgTL50/tQxNi+cyN6fvyFxyx5Cd5+n3GmoCpRNKbUSVoXKV3WgSe+7aV61ntchG2PSsARj8oUjO1azed4HHFm1iqDtR6i6P5niib5SK4fLwaG6JTjZtA61Ot1Aq+tut1IrxhQAlmBMnktOSuK3n2ax87s5nNkQQ5mdvlIr5YGQADhQVdh9VXnKhbWkYc/badq8c5Z9GmPyH0swxnVnj8exed577P/1e5K37qdybAIhZ6EacDql1Eq7K6jS7lqa9LqHlqHVvQ7ZGJMLLMGYXBe35Re2zp/MsejVFP/tGFfsV4KTfb89OVgBDjRKKbXSj9bXDLJSK8YUUpZgzGVJSjjPtu8/ZtcPX3Ju42+U2+UrtVIRX6mV/dUC2HVNBcq3Dqdx77toWr+t1yEbY/KIJRhzSU4d3s3mr98jbtlSiDlAlT1pSq3UDOL0tTWo1uE6mvS6h1ZWasWYIssSjMnU3jXfOKVW1vlKrRxQSqmvsvCBUNjXvDSlWzSiTpdbady+v5VaMcaksgRjUiWcPUXMd1PY/eN8zm/aSYXd8VQ84ZRaKQYHqgew87pKhLaJoHGfkTSt1dzrkI0x+ZglmCLs+N7NbJ73AYdW/ErA9kNU2ZtEyfO+8tbHQuBIreKc7laL6lf3oFnPEYRbqRVjzCWwBFNEJCclEbtqLtsXz+bU2o2U2nmKqn6lVg5UFvaGlSGkZTMa9IikcVhPG+4yxlwWSzCF1Pkzx9m84AP2/ryIxM2xVNx9nvL+pVaqB7KzVSUqRTilVqo18DpkY0whYwmmkDi6cy2b533A4VUrKLb9CFX3JRPslFo5UhYO1y3BqSZXUrNjb1p1G26lVowxrrMEUwAlJyWx4+dP2fHdZ5xZH0PIrtNUPeybDrS0U2olNqIcZcNa0rDnH2jaoovXIRtjiiBLMAXA2eNxbJ4/kf2/fEfy1n1Uik2gTEqplWCn1EpEFaq2u5ZGve+mZWitLPs0xhi3uZpgRKQ38Aa+GS3fV9WX0mwPBqYAbYHDwBBV3SEifwAe9WvaCmgDbAN+8FtfE/hIVf8sIncCr+CbVhngbVV9P/fflfsObl3GlvmTfKVWth+j6oFkgpOcUivlIa5RSU42rU/tzjfRutNQK7VijMmXXEswIhIIjAd6ArHAchGJUtUNfs1GAkdVtYGIRAJj8SWZj4GPnX5aAv9V1WjnNeF++1gJfObX3wxVHe3We3JDUsJ5tv8wjZ3ff8G5Tdspu/Mslf1KrRyoGsDuqytSPjyMRr3upGnDdl6HbIwx2eLmGUw7IEZVtwOIyHSgH+CfYPoBzzjLs4G3RUT0wnmchwLT0nYuIg2BKlx4RpPvnT68l83z3uXAsh9haxyV9yRS2im1crIkHKoVxJlrqnFFhy407jWSVuWqeB2yMcbkiJsJpgaw2+95LNA+ozaqmigix4FQ4JBfmyH4ElFaQ/GdsfgnowEi0hnYAvxFVXen87o8tW/dd2xd+DEnVq+lxI4TVI1TSib/Xmplf7PSlGrekDpd+tP46gH22xNjTKHhZoKRdNbppbQRkfbAGVVdl067SGCY3/MvgGmqGi8i9wGTgW4XBSUyChgFULt27UzfwKVKjD/L1sWTif3xa+I376LCrnOppVbKFXMqC3cKJbRNhG+4q05Yru7fGGPyEzcTTCy+qiMpagJ7M2gTKyLF8N1pe8RveyTpD4+FAcVUdWXKOlU97NfkPXzXcy6iqhOACQARERFpE94lOXlgOxvnTuDQil8I2PZ7qZWawLHSvlIrp7rWpPrVPWjaYwRhIeUvZ3fGGFOguJlglgMNRaQuvju7IoHb0rSJAoYDPwMDgcUpQ14iEgAMAtKbL/ei6zIiUk1V9zlP+wIbc+l9XGTJq3eRGPWLr9SKQmmcUiutyhDSsin1ug2kcZsbbLjLGFOkuZZgnGsqo4H5+G5T/kBV14vIc8AKVY0CJgJTRSQG35lLpF8XnYHYlJsE0hgM3JBm3UMi0hdIdPq6M1ffkL+AAM6XCmBn10pUimhP49530bx6Y9d2Z4wxBZFceI28aImIiNAVK1Z4HYYxxhQoIrJSVSOyaheQF8EYY4wpeizBGGOMcYUlGGOMMa6wBGOMMcYVlmCMMca4whKMMcYYV1iCMcYY4wpLMMYYY1xRpH9oKSIHgZ05fHklLqz6nF9YXJfG4rp0+TU2i+vSXE5cV6pq5awaFekEczlEZEV2fsma1yyuS2NxXbr8GpvFdWnyIi4bIjPGGOMKSzDGGGNcYQkm5yZ4HUAGLK5LY3Fduvwam8V1aVyPy67BGGOMcYWdwRhjjHGFJZhMiMgHIhInIusy2C4i8qaIxIjIGhFpk0/i6iIix0Uk2nk8lUdx1RKRb0Vko4isF5E/pdMmz49ZNuPK82MmIiVEZJmIrHbiejadNsEiMsM5Xr+KSJ18EtedInLQ73jd7XZcfvsOFJH/iciX6WzL8+OVzbi8PF47RGSts9+LJsBy9d+kqtojgwe+WTXbAOsy2H4D8DUgQAfg13wSVxfgSw+OVzWgjbNcBtgCNPP6mGUzrjw/Zs4xCHGWg4BfgQ5p2jwA/J+zHAnMyCdx3Qm8ndf/jzn7fhj4JL2/lxfHK5txeXm8dgCVMtnu2r9JO4PJhKouwTf9ckb6AVPU5xegvIhUywdxeUJV96nqKmf5JLARqJGmWZ4fs2zGleecY3DKeRrkPNJeFO0HTHaWZwPdRUTyQVyeEJGawI3A+xk0yfPjlc248jPX/k1agrk8NYDdfs9jyQcfXI6rnSGOr0WkeV7v3BmaaI3v268/T49ZJnGBB8fMGVaJBuKAhaqa4fFS1UTgOBCaD+ICGOAMqcwWkVpux+QYB/wNSM5guyfHKxtxgTfHC3xfDhaIyEoRGZXOdtf+TVqCuTzpfTPKD9/0VuEr5RAGvAV8npc7F5EQ4FPgz6p6Iu3mdF6SJ8csi7g8OWaqmqSq4UBNoJ2ItEjTxJPjlY24vgDqqGorYBG/nzW4RkRuAuJUdWVmzdJZ5+rxymZceX68/HRU1TZAH+BBEemcZrtrx8wSzOWJBfy/idQE9noUSypVPZEyxKGqc4EgEamUF/sWkSB8H+Ifq+pn6TTx5JhlFZeXx8zZ5zHgO6B3mk2px0tEigHlyMPh0YziUtXDqhrvPH0PaJsH4XQE+orIDmA60E1EPkrTxovjlWVcHh2vlH3vdf4bB8wB2qVp4tq/SUswlycKuMO5C6MDcFxV93kdlIhckTLuLCLt8P2dD+fBfgWYCGxU1dcyaJbnxyw7cXlxzESksoiUd5ZLAj2ATWmaRQHDneWBwGJ1rsx6GVeaMfq++K5ruUpVH1fVmqpaB98F/MWqenuaZnl+vLITlxfHy9lvaREpk7IMXA+kvfvUtX+TxXKjk8JKRKbhu7uokojEAk/ju+CJqv4fMBffHRgxwBlgRD6JayBwv4gkAmeBSLf/kTk6AsOAtc74PcATQG2/2Lw4ZtmJy4tjVg2YLCKB+BLaTFX9UkSeA1aoahS+xDhVRGLwfROPdDmm7Mb1kIj0BRKduO7Mg7jSlQ+OV3bi8up4VQXmON+digGfqOo8EbkP3P83ab/kN8YY4wobIjPGGOMKSzDGGGNcYQnGGGOMKyzBGGOMcYUlGGOMMa6wBGOMMcYVlmCMKQCckus5qizglIqvnht9GXMpLMEYU/jdCVTPqpExuc0SjDGXQETqiMgmEXlfRNaJyMci0kNElorIVhFp5zx+Et/kUz+JSGPntQ+LyAfOckvn9aUy2E+oiCxw+ngXv4KEInK7+CYEixaRd51f3CMip0Tk3yKySkS+cUq+DAQigI+d9iWdbv7otFsrIk3cPGam6LIEY8ylawC8AbQCmgC3AdcCj+ArQbMJ6KyqrYGngBec140DGohIf+BD4F5VPZPBPp4GfnT6iMIpayMiTYEh+CrkhgNJwB+c15QGVjmVc78HnlbV2cAK4A+qGq6qZ522h5x27zhxG5PrrBaZMZfuN1VdCyAi64FvVFVFZC1QB18F38ki0hBf2fOUOnHJInInsAZ4V1WXZrKPzsCtzuu+EpGjzvru+CrxLnfqS5XEN2cL+OYimeEsfwSkV806Rcq2lSn7MSa3WYIx5tLF+y0n+z1Pxvdv6p/At6raX3wTnH3n174hcIrsXRNJr1CgAJNV9fEcvj5FSsxJ2OeAcYkNkRmT+8oBe5zlO1NWikg5fENrnYFQ5/pIRpbgDH2JSB+ggrP+G2CgiFRxtlUUkSudbQH4qkKDb9juR2f5JFDmMt6PMTliCcaY3Pcy8KKILAUC/da/DvxHVbcAI4GXUhJFOp4FOovIKnxzeOwCUNUNwN/xTYG7BliIr7w+wGmguYisBLoBzznrJwH/l+YivzGus3L9xhQSInJKVUO8jsOYFHYGY4wxxhV2BmOMh0RkBPCnNKuXquqDXsRjTG6yBGOMMcYVNkRmjDHGFZZgjDHGuMISjDHGGFdYgjHGGOMKSzDGGGNc8f8+QJ+2lkMJPAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
